import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import pickle

from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error

# Pick only relevant columns for the analysis
columns_needed = ['Site', 'Charging event', 'Date', 'Total kWh']

# Load datasets and merge into one
df_1 = pd.read_csv('charge-sessions-june-sept.csv').drop(columns=['Unnamed: 13', 'Unnamed: 14'])
df_1['Date'] = pd.to_datetime(df_1['Start Date'], format="%d/%m/%Y")
df_1 = df_1[columns_needed]

df_2 = pd.read_csv('cp-data-dec-mar-2018.csv')
df_2['Date'] = pd.to_datetime(df_2['Start Date'], format="%d/%m/%Y")
df_2 = df_2[columns_needed]

df_3 = pd.read_csv('cp-data-mar-may-2018.csv')
df_3['Date'] = pd.to_datetime(df_3['Start Date'], format="%d/%m/%Y")
df_3 = df_3[columns_needed]

# Union data
df = pd.concat([df_1, df_2, df_3]).drop_duplicates()
df = df[df['Site'] != '***TEST SITE*** Charge Your Car HQ']
df = df[df['Site'] != 'Dundee House, Dundee']
df = df[df['Site'] != 'Princes Street Charging Hub']

# Define a mapping dictionary for renaming
rename_mapping = {
    'Sinclair Street, Dundee': 'Sinclair Street',
    'South Tay Street, Dundee': 'South Tay Street',
    'Trades Lane, Dundee': 'Trades Lane',
}

df['Site'] = df['Site'].replace(rename_mapping)
df.head()

# Construct dictionary for storing the datasets
stations = df['Site'].unique()
stations_dict = {}

for station in stations:
    stations_dict[station] = {'data':pd.DataFrame(), 'daily':pd.DataFrame(), 'weekly':pd.DataFrame(), 'monthly':pd.DataFrame()}
    stations_dict[station]['data'] = df[df['Site'] == station]

print('STATION NAMES: \n')

for station in stations:
    print(station)
    df_temp = stations_dict[station]['data'].copy()
    
    df_temp['Date'] = pd.to_datetime(df_temp['Date'])
    df_temp['Year-Month'] = df_temp['Date'].dt.strftime('%Y-%m')

    # Define the start date and end date
    start_date = df_temp['Date'].min()
    end_date = df_temp['Date'].max()

    # Calculate the number of weeks from the start date for each date
    df_temp['Week_index'] = (df_temp['Date'] - start_date).dt.days // 7

    stations_dict[station]['data'] = df_temp

STATION NAMES: 

Lochee Charging Hub, Dundee
Greenmarket Multi Car Park, Dundee
Queen Street Car Park, Broughty Ferry, Dundee
Housing Office West, Dundee
Nethergate, Dundee
Brington Place Sheltered Housing, Dundee
Balunie Drive, Dundee
Social Work Building, Jack Martin Way, Dundee
Dundee Ice Arena, Dundee
Mitchell Street, Dundee
Oakland Day Centre, Dundee
Dock Street, Dundee
Whitfield Centre, Dundee
Housing Office East, Dundee
Gellatly Street Car Park, Dundee
Public Works Dept, Clepington Rd. Dundee
Marchbanks, Dundee
Olympia Multi-Storey Car Park, Dundee
South Tay Street
Ardler Complex, Dundee
Menziehill House, Dundee
Turriff House Rannoch Road, Dundee
Trades Lane
University of Dundee, Nethergate, Dundee
Janet Brougham House, Dundee
DCC Environment, 34 Harefield Road
Sinclair Street
Earn Cresent, Dundee

for station in stations:

    df_temp = stations_dict[station]['data'].copy()
    daily = df_temp.groupby(['Date', 'Site'])['Total kWh'].sum().reset_index()

    # Define the complete date range
    start_date = df_temp['Date'].min()
    end_date = df_temp['Date'].max()
    date_range = pd.date_range(start=start_date, end=end_date)

    station_data = pd.DataFrame({'Date': date_range, 'Site': station})

    # Merge the complete data with the daily DataFrame to fill in missing values with 0
    daily_merged = pd.merge(station_data, daily, on=['Date', 'Site'], how='left')

    # Use mean for filling missing data 
    mean_energy = daily_merged['Total kWh'].mean()
    rolling_mean_energy = daily_merged['Total kWh'].rolling(window=5).mean().shift(1).fillna(mean_energy)
    daily_merged['Total kWh'] = daily_merged['Total kWh'].fillna(rolling_mean_energy)

    stations_dict[station]['daily'] = daily_merged

for station in stations:

    df_temp = stations_dict[station]['data'].copy()
    
    # Group by Week_index and Station Name to get weekly energy consumption
    weekly = df_temp.groupby(['Week_index', 'Site'])['Total kWh'].sum().reset_index()

    # Define the complete week index range
    start_week = df_temp['Week_index'].min()
    end_week = df_temp['Week_index'].max()
    week_range = range(start_week, end_week + 1)

    station_data = pd.DataFrame({'Week_index': week_range, 'Site': station})

    # Merge the complete data with the weekly DataFrame to fill in missing values with 0
    weekly_merged = pd.merge(station_data, weekly, on=['Week_index', 'Site'], how='left')

    # Use mean for filling missing data 
    mean_energy = weekly_merged['Total kWh'].mean()
    rolling_mean_energy = weekly_merged['Total kWh'].rolling(window=5).mean().shift(1).fillna(mean_energy)
    weekly_merged['Total kWh'] = weekly_merged['Total kWh'].fillna(rolling_mean_energy)

    stations_dict[station]['weekly'] = weekly_merged

# Define the number of rows and columns for the subplots
num_rows = 7
num_cols = 4

# Create a figure and subplots
fig, axs = plt.subplots(num_rows, num_cols, figsize=(35, 35))

# Flatten the axes for easier iteration
axs = axs.flatten()

# Iterate over each station and plot on a separate subplot
for i, station in enumerate(stations):
    plot_df = stations_dict[station]['daily']
    ax = axs[i]
    ax.plot(plot_df['Date'], plot_df['Total kWh'])
    ax.set_title(station)
    ax.set_xlabel('Date')
    ax.set_ylabel('Energy (kWh)')
    ax.grid(True)
    plt.setp(ax.get_xticklabels(), rotation=45)

# Adjust layout
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Energy demand day by day for different stations', fontsize=30)
plt.show()

class LSTMModel(nn.Module):
      # input_size : number of features in input at each time step
      # hidden_size : Number of LSTM units 
      # num_layers : number of LSTM layers 
    def __init__(self, input_size, hidden_size, num_layers): 
        super(LSTMModel, self).__init__() #initializes the parent class nn.Module
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, dropout=0.5)
        self.linear = nn.Linear(hidden_size, 1)
 
    def forward(self, x): # defines forward pass of the neural network
        out, _ = self.lstm(x)
        # extract only the last time step
        out = out[:, -1, :]
        out = self.linear(out)
        return out

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)

cuda

def train_lstm(df_dict, type, sequence_length, stations, num_epochs, predictions_dict, test_obs):
    for station in stations:
        print('\n', station, '\n')
        temp_df = df_dict[station][type]['Total kWh']

        # Scaling dataset
        reshaped_data = np.reshape(temp_df, (-1,1))
        scaler = MinMaxScaler(feature_range=(0,1))
        scaled_data = scaler.fit_transform(reshaped_data)

        # Create sequences and labels for training data
        sequence_length = sequence_length  # Number of time steps to look back
        X, y = [], []
        for i in range(len(scaled_data) - sequence_length):
            X.append(scaled_data[i:i+sequence_length])
            y.append(scaled_data[i+sequence_length])
        X, y = np.array(X), np.array(y)

        # Splitting the dataset
        X_train = X[:-test_obs]
        y_train = y[:-test_obs]
        X_test = X[-test_obs:]
        y_test = y[-test_obs:]
        print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
    
        # Convert data to PyTorch tensors
        X_train = torch.tensor(X_train, dtype=torch.float32)
        y_train = torch.tensor(y_train, dtype=torch.float32)
        X_test = torch.tensor(X_test, dtype=torch.float32)
        y_test = torch.tensor(y_test, dtype=torch.float32)
        X_train.shape,y_train.shape

        input_size = 1
        num_layers = 1
        hidden_size = 64
        
        # Define the model, loss function, and optimizer
        model = LSTMModel(input_size, hidden_size, num_layers).to(device)
        
        loss_fn = torch.nn.MSELoss(reduction='mean')
        
        optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

        batch_size = 15
        # Create DataLoader for batch training
        train_dataset = torch.utils.data.TensorDataset(X_train, y_train)
        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False)
        
        # Create DataLoader for batch training
        test_dataset = torch.utils.data.TensorDataset(X_test, y_test)
        test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

        num_epochs = num_epochs
        train_hist = []
        test_hist = []
        # Training loop
        for epoch in range(num_epochs):
            total_loss = 0.0

            # Training
            model.train()
            for batch_X, batch_y in train_loader:
                batch_X, batch_y = batch_X.to(device), batch_y.to(device)
                predictions = model(batch_X)
                loss = loss_fn(predictions, batch_y)

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                total_loss += loss.item()

            # Calculate average training loss and accuracy
            average_loss = total_loss / len(train_loader)
            train_hist.append(average_loss)

            # Validation on test data
            model.eval()
            with torch.no_grad():
                total_test_loss = 0.0

                for batch_X_test, batch_y_test in test_loader:
                    batch_X_test, batch_y_test = batch_X_test.to(device), batch_y_test.to(device)
                    predictions_test = model(batch_X_test)
                    test_loss = loss_fn(predictions_test, batch_y_test)

                    total_test_loss += test_loss.item()

                # Calculate average test loss and accuracy
                average_test_loss = total_test_loss / len(test_loader)
                test_hist.append(average_test_loss)
            if (epoch+1)%10==0:
                print(f'Epoch [{epoch+1}/{num_epochs}] - Training Loss: {average_loss:.4f}, Test Loss: {average_test_loss:.4f}')

        #training plot
        x = np.linspace(1,num_epochs,num_epochs)
        plt.plot(x, train_hist, scalex=True, label="Training loss")
        plt.plot(x, test_hist, label="Test loss")
        plt.suptitle(station)
        plt.legend()
        plt.show()

        # # Save the model's state dictionary to a file
        # if station == 'HAMILTON':
        #     torch.save(model.state_dict(), f'lstm_model_weekly_{station}.pth')

        # # Save the model's state dictionary to a file
        # if station == 'MPL':
        #     torch.save(model.state_dict(), f'lstm_model_weekly_{station}.pth')
        
        # Set the model to evaluation mode
        model.eval()

        # Move input tensor to the same device as model's parameters
        X_test = X_test.to(device)

        # Pass the entire test set through the model to obtain predictions
        with torch.no_grad():
            # Obtain predictions for the entire test set
            predictions_test = model(X_test)
            
            # Convert predictions from tensors to NumPy arrays
            predictions_test = predictions_test.cpu().numpy()

        # Reverse predictions
        y_test_reversed = scaler.inverse_transform(y_test.reshape(-1, 1))
        predictions_reversed = scaler.inverse_transform(predictions_test.reshape(-1, 1))

        # Save y_test and predictions
        predictions_dict[station]['y_pred'] = predictions_reversed
        predictions_dict[station]['y_test'] = y_test_reversed

        # Calculate mean squared error
        mse = mean_squared_error(y_test_reversed, predictions_reversed)

        # Calculate root mean squared error
        rmse = np.sqrt(mse)

        print(f"{station} Root Mean Squared Error (RMSE):", rmse)
        predictions_dict[station]['rmse'] = rmse

# Initialize dict for storing predictions
daily_predictions_dict = {}
for station in stations:
    daily_predictions_dict[station] = {'y_test':[], 'y_pred':[], 'rmse':0}

train_lstm(df_dict=stations_dict, type='daily', sequence_length=30, stations=stations, 
           num_epochs=50, predictions_dict=daily_predictions_dict, test_obs=30)

 Lochee Charging Hub, Dundee 

(90, 30, 1) (90, 1) (30, 30, 1) (30, 1)

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [10/50] - Training Loss: 0.0300, Test Loss: 0.0216
Epoch [20/50] - Training Loss: 0.0303, Test Loss: 0.0217
Epoch [30/50] - Training Loss: 0.0301, Test Loss: 0.0217
Epoch [40/50] - Training Loss: 0.0299, Test Loss: 0.0217
Epoch [50/50] - Training Loss: 0.0296, Test Loss: 0.0216

Lochee Charging Hub, Dundee Root Mean Squared Error (RMSE): 164.9985884652239

 Greenmarket Multi Car Park, Dundee 

(219, 30, 1) (219, 1) (30, 30, 1) (30, 1)

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [10/50] - Training Loss: 0.0233, Test Loss: 0.0132
Epoch [20/50] - Training Loss: 0.0192, Test Loss: 0.0100
Epoch [30/50] - Training Loss: 0.0182, Test Loss: 0.0097
Epoch [40/50] - Training Loss: 0.0178, Test Loss: 0.0097
Epoch [50/50] - Training Loss: 0.0175, Test Loss: 0.0098

Greenmarket Multi Car Park, Dundee Root Mean Squared Error (RMSE): 40.68734405712979

 Queen Street Car Park, Broughty Ferry, Dundee 

(219, 30, 1) (219, 1) (30, 30, 1) (30, 1)

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [10/50] - Training Loss: 0.0308, Test Loss: 0.0236
Epoch [20/50] - Training Loss: 0.0281, Test Loss: 0.0249
Epoch [30/50] - Training Loss: 0.0252, Test Loss: 0.0240
Epoch [40/50] - Training Loss: 0.0254, Test Loss: 0.0250
Epoch [50/50] - Training Loss: 0.0248, Test Loss: 0.0243

Queen Street Car Park, Broughty Ferry, Dundee Root Mean Squared Error (RMSE): 49.33874486690207

 Housing Office West, Dundee 

(219, 30, 1) (219, 1) (30, 30, 1) (30, 1)

# File path to save the dictionary
file_path = 'daily_predictions_dundee.pkl'

# Save the dictionary to a file using pickle
with open(file_path, 'wb') as file:
    pickle.dump(daily_predictions_dict, file)

print(f"Dictionary saved to '{file_path}'")

Dictionary saved to 'daily_predictions_dundee.pkl'

# Define the number of rows and columns for subplots
n_rows = 10
n_cols = 3

# Create subplots
fig, axs = plt.subplots(n_rows, n_cols, figsize=(20, 40))

# Flatten the axis array for easier indexing
axs = axs.flatten()

# Iterate over each station
for i, station in enumerate(daily_predictions_dict.keys()):
    # Get data for the current station
    station_data = daily_predictions_dict[station]
    y_test = station_data['y_test']
    y_pred = station_data['y_pred']
    rmse = station_data['rmse']

    # Plot y_pred against y_test
    axs[i].scatter(y_test, y_pred, color='blue', label='Predicted vs Actual', s=60)
    axs[i].plot(y_test, y_test, color='red', linestyle='--', label='Ideal line', linewidth=3)

    axs[i].set_title(f'{station.split()[0]} - RMSE: {rmse:.2f}', fontsize=23)
    axs[i].set_xlabel('Actual (y_test)', fontsize=15)
    axs[i].set_ylabel('Predicted (y_pred)', fontsize=15)
    axs[i].legend(fontsize=15)

# Adjust layout and show plot
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Scatter plots comparing daily predictions and actual values for every station', fontsize=40)
plt.show()

# Define the number of rows and columns for subplots
n_rows = 10
n_cols = 3

# Create subplots
fig, axs = plt.subplots(n_rows, n_cols, figsize=(20, 40))

# Flatten the axis array for easier indexing
axs = axs.flatten()

# Iterate over each station
for i, station in enumerate(daily_predictions_dict.keys()):
    # Get data for the current station
    station_data = daily_predictions_dict[station]
    y_test = station_data['y_test']
    y_pred = station_data['y_pred']
    rmse = station_data['rmse']

    # Plot true values and predicted values against the index of observations
    x = np.arange(len(y_test))
    axs[i].plot(x, y_test, color='blue', label='True values', linewidth=3)
    axs[i].plot(x, y_pred, color='red', linestyle='--', label='Predicted values', linewidth=3)

    axs[i].set_title(f'{station.split()[0]} - RMSE: {rmse:.2f}', fontsize=27)
    axs[i].set_xlabel('Observation Index', fontsize=20)
    axs[i].set_ylabel('Value', fontsize=20)
    axs[i].legend(fontsize=15)

# Adjust layout and show plot
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Line plots comparing daily predictions and actual values for every station', fontsize=40)
plt.show()

# Initialize dict for storing predictions
weekly_predictions_dict = {}
for station in stations:
    weekly_predictions_dict[station] = {'y_test':[], 'y_pred':[], 'rmse':0}

stations_dict_temp = stations_dict.copy()
stations_dict_temp.pop('DCC Environment, 34 Harefield Road', None)
stations_dict_temp.pop('Sinclair Street', None)
stations_dict_temp.pop('Earn Cresent, Dundee', None)

stations_temp = list(stations.copy())
stations_temp.remove('DCC Environment, 34 Harefield Road')
stations_temp.remove('Sinclair Street')
stations_temp.remove('Earn Cresent, Dundee')

train_lstm(df_dict=stations_dict_temp, type='weekly', sequence_length=5, stations=stations_temp, 
           num_epochs=100, predictions_dict=weekly_predictions_dict, test_obs=12)

 Lochee Charging Hub, Dundee 

(5, 5, 1) (5, 1) (12, 5, 1) (12, 1)
Epoch [10/100] - Training Loss: 0.3251, Test Loss: 0.2133
Epoch [20/100] - Training Loss: 0.1461, Test Loss: 0.0896
Epoch [30/100] - Training Loss: 0.0180, Test Loss: 0.0959
Epoch [40/100] - Training Loss: 0.0239, Test Loss: 0.0927

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [50/100] - Training Loss: 0.0220, Test Loss: 0.0589
Epoch [60/100] - Training Loss: 0.0178, Test Loss: 0.0731
Epoch [70/100] - Training Loss: 0.0184, Test Loss: 0.0807
Epoch [80/100] - Training Loss: 0.0178, Test Loss: 0.0702
Epoch [90/100] - Training Loss: 0.0177, Test Loss: 0.0736
Epoch [100/100] - Training Loss: 0.0177, Test Loss: 0.0750

Lochee Charging Hub, Dundee Root Mean Squared Error (RMSE): 1410.1596638025185

 Greenmarket Multi Car Park, Dundee 

(23, 5, 1) (23, 1) (12, 5, 1) (12, 1)
Epoch [10/100] - Training Loss: 0.0414, Test Loss: 0.0362
Epoch [20/100] - Training Loss: 0.0336, Test Loss: 0.0594
Epoch [30/100] - Training Loss: 0.0317, Test Loss: 0.0426

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [40/100] - Training Loss: 0.0302, Test Loss: 0.0428
Epoch [50/100] - Training Loss: 0.0288, Test Loss: 0.0373
Epoch [60/100] - Training Loss: 0.0275, Test Loss: 0.0328
Epoch [70/100] - Training Loss: 0.0263, Test Loss: 0.0280
Epoch [80/100] - Training Loss: 0.0254, Test Loss: 0.0240
Epoch [90/100] - Training Loss: 0.0248, Test Loss: 0.0212
Epoch [100/100] - Training Loss: 0.0243, Test Loss: 0.0199

Greenmarket Multi Car Park, Dundee Root Mean Squared Error (RMSE): 255.5779040304386

 Queen Street Car Park, Broughty Ferry, Dundee 

(23, 5, 1) (23, 1) (12, 5, 1) (12, 1)
Epoch [10/100] - Training Loss: 0.0447, Test Loss: 0.0398
Epoch [20/100] - Training Loss: 0.0355, Test Loss: 0.0570

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Epoch [30/100] - Training Loss: 0.0346, Test Loss: 0.0475
Epoch [40/100] - Training Loss: 0.0325, Test Loss: 0.0458
Epoch [50/100] - Training Loss: 0.0306, Test Loss: 0.0395
Epoch [60/100] - Training Loss: 0.0282, Test Loss: 0.0317
Epoch [70/100] - Training Loss: 0.0260, Test Loss: 0.0232
Epoch [80/100] - Training Loss: 0.0253, Test Loss: 0.0176
Epoch [90/100] - Training Loss: 0.0252, Test Loss: 0.0172
Epoch [100/100] - Training Loss: 0.0251, Test Loss: 0.0179

Queen Street Car Park, Broughty Ferry, Dundee Root Mean Squared Error (RMSE): 133.23601466232165

 Housing Office West, Dundee 

(23, 5, 1) (23, 1) (12, 5, 1) (12, 1)
Epoch [10/100] - Training Loss: 0.0615, Test Loss: 0.0340
Epoch [20/100] - Training Loss: 0.0472, Test Loss: 0.0255

weekly_predictions_dict.pop('DCC Environment, 34 Harefield Road', None)
weekly_predictions_dict.pop('Sinclair Street', None)
weekly_predictions_dict.pop('Earn Cresent, Dundee', None)

{'y_test': [], 'y_pred': [], 'rmse': 0}

# File path to save the dictionary
file_path = 'weekly_predictions_dundee.pkl'

# Save the dictionary to a file using pickle
with open(file_path, 'wb') as file:
    pickle.dump(weekly_predictions_dict, file)

print(f"Dictionary saved to '{file_path}'")

Dictionary saved to 'weekly_predictions_dundee.pkl'

# Define the number of rows and columns for subplots
n_rows = 9
n_cols = 3

# Create subplots
fig, axs = plt.subplots(n_rows, n_cols, figsize=(20, 40))

# Flatten the axis array for easier indexing
axs = axs.flatten()

# Iterate over each station
for i, station in enumerate(weekly_predictions_dict.keys()):
    # Get data for the current station
    station_data = weekly_predictions_dict[station]
    y_test = station_data['y_test']
    y_pred = station_data['y_pred']
    rmse = station_data['rmse']

    # Plot y_pred against y_test
    axs[i].scatter(y_test, y_pred, color='blue', label='Predicted vs Actual', s=60)
    axs[i].plot(y_test, y_test, color='red', linestyle='--', label='Ideal line', linewidth=3)

    axs[i].set_title(f'{station.split()[0]} - RMSE: {rmse:.2f}', fontsize=23)
    axs[i].set_xlabel('Actual (y_test)', fontsize=15)
    axs[i].set_ylabel('Predicted (y_pred)', fontsize=15)
    axs[i].legend(fontsize=15)

# Adjust layout and show plot
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Scatter plots comparing weekly predictions and actual values for every station', fontsize=40)
plt.show()

# Define the number of rows and columns for subplots
n_rows = 9
n_cols = 3

# Create subplots
fig, axs = plt.subplots(n_rows, n_cols, figsize=(20, 40))

# Flatten the axis array for easier indexing
axs = axs.flatten()

# Iterate over each station
for i, station in enumerate(weekly_predictions_dict.keys()):
    # Get data for the current station
    station_data = weekly_predictions_dict[station]
    y_test = station_data['y_test']
    y_pred = station_data['y_pred']
    rmse = station_data['rmse']

    # Plot true values and predicted values against the index of observations
    x = np.arange(len(y_test))
    axs[i].plot(x, y_test, color='blue', label='True values', linewidth=3)
    axs[i].plot(x, y_pred, color='red', linestyle='--', label='Predicted values', linewidth=3)

    axs[i].set_title(f'{station.split()[0]} - RMSE: {rmse:.2f}', fontsize=27)
    axs[i].set_xlabel('Observation Index', fontsize=20)
    axs[i].set_ylabel('Value', fontsize=20)
    axs[i].legend(fontsize=15)

# Adjust layout and show plot
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Line plots comparing weekly predictions and actual values for every station', fontsize=40)
plt.show()

def process_data(df_dict, type, sequence_length, stations, predictions_dict, test_obs):
    for station in stations:
        temp_df = df_dict[station][type]['Total kWh']

        # Scaling dataset
        reshaped_data = np.reshape(temp_df, (-1,1))
        scaler = MinMaxScaler(feature_range=(0,1))
        scaled_data = scaler.fit_transform(reshaped_data)

        # Create sequences and labels for training data
        sequence_length = sequence_length  # Number of time steps to look back
        X, y = [], []
        for i in range(len(scaled_data) - sequence_length):
            X.append(scaled_data[i:i+sequence_length])
            y.append(scaled_data[i+sequence_length])
        X, y = np.array(X), np.array(y)

        # Splitting the dataset
        X_test = X[-test_obs:]
        y_test = y[-test_obs:]
    
        # Convert data to PyTorch tensors
        X_test = torch.tensor(X_test, dtype=torch.float32)
        y_test = torch.tensor(y_test, dtype=torch.float32)

        predictions_dict[station]['X_test'] = X_test        
        predictions_dict[station]['y_test'] = y_test 
        predictions_dict[station]['scaler'] = scaler

# Initialize dict for storing predictions
weekly_predictions_dict_transfer = {}
for station in stations:
    weekly_predictions_dict_transfer[station] = {'scaler': None, 'X_test': [], 'y_test':[], 'y_pred':[], 'rmse':0}

weekly_predictions_dict_transfer.pop('DCC Environment, 34 Harefield Road', None)
weekly_predictions_dict_transfer.pop('Sinclair Street', None)
weekly_predictions_dict_transfer.pop('Earn Cresent, Dundee', None)
weekly_predictions_dict_transfer.pop('Lochee Charging Hub, Dundee', None)
weekly_predictions_dict_transfer.pop('University of Dundee, Nethergate, Dundee', None)
weekly_predictions_dict_transfer.pop('Janet Brougham House, Dundee', None)
weekly_predictions_dict_transfer.pop('Trades Lane', None)

{'scaler': None, 'X_test': [], 'y_test': [], 'y_pred': [], 'rmse': 0}

stations_dict_temp = stations_dict.copy()
stations_dict_temp.pop('DCC Environment, 34 Harefield Road', None)
stations_dict_temp.pop('Sinclair Street', None)
stations_dict_temp.pop('Earn Cresent, Dundee', None)
stations_dict_temp.pop('Lochee Charging Hub, Dundee', None)
stations_dict_temp.pop('University of Dundee, Nethergate, Dundee', None)
stations_dict_temp.pop('Janet Brougham House, Dundee', None)
stations_dict_temp.pop('Trades Lane', None)

stations_temp = list(stations.copy())
stations_temp.remove('DCC Environment, 34 Harefield Road')
stations_temp.remove('Sinclair Street')
stations_temp.remove('Earn Cresent, Dundee')
stations_temp.remove('Lochee Charging Hub, Dundee')
stations_temp.remove('University of Dundee, Nethergate, Dundee')
stations_temp.remove('Janet Brougham House, Dundee')
stations_temp.remove('Trades Lane')

process_data(df_dict=stations_dict_temp, type='weekly', sequence_length=30, stations=stations_temp, 
             predictions_dict=weekly_predictions_dict_transfer, test_obs=12)

predictions_dict = weekly_predictions_dict_transfer

for station in stations_temp:
    scaler = weekly_predictions_dict_transfer[station]['scaler']

    input_size = 1
    num_layers = 1
    hidden_size = 64
            
    # Reinitalize the model class
    model = LSTMModel(input_size, hidden_size, num_layers).to(device)

    # Load model
    model.load_state_dict(torch.load("lstm_model_weekly_MPL.pth"))
    model.eval()  # Set the model to evaluation mode

    # Move input tensor to the same device as model's parameters
    X_test = weekly_predictions_dict_transfer[station]['X_test'].to(device)
    y_test = weekly_predictions_dict_transfer[station]['y_test']

    # Pass the entire test set through the model to obtain predictions
    with torch.no_grad():
        # Obtain predictions for the entire test set
        predictions_test = model(X_test)
        
        # Convert predictions from tensors to NumPy arrays
        predictions_test = predictions_test.cpu().numpy()

    # Reverse predictions
    y_test_reversed = scaler.inverse_transform(y_test.reshape(-1, 1))
    predictions_reversed = scaler.inverse_transform(predictions_test.reshape(-1, 1))

    # Save y_test and predictions
    predictions_dict[station]['y_pred'] = predictions_reversed
    predictions_dict[station]['y_test'] = y_test_reversed

    # Calculate mean squared error
    mse = mean_squared_error(y_test_reversed, predictions_reversed)

    # Calculate root mean squared error
    rmse = np.sqrt(mse)

    print(f"{station} Root Mean Squared Error (RMSE):", rmse)
    predictions_dict[station]['rmse'] = rmse

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "
c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

Greenmarket Multi Car Park, Dundee Root Mean Squared Error (RMSE): 180.28937555401123
Queen Street Car Park, Broughty Ferry, Dundee Root Mean Squared Error (RMSE): 109.51037015058755
Housing Office West, Dundee Root Mean Squared Error (RMSE): 28.21207088394768
Nethergate, Dundee Root Mean Squared Error (RMSE): 38.084885651980336
Brington Place Sheltered Housing, Dundee Root Mean Squared Error (RMSE): 16.83455976046998
Balunie Drive, Dundee Root Mean Squared Error (RMSE): 30.21763366880114
Social Work Building, Jack Martin Way, Dundee Root Mean Squared Error (RMSE): 20.802249559672287
Dundee Ice Arena, Dundee Root Mean Squared Error (RMSE): 229.6177991041813
Mitchell Street, Dundee Root Mean Squared Error (RMSE): 8.443729630277652
Oakland Day Centre, Dundee Root Mean Squared Error (RMSE): 38.02191604131285
Dock Street, Dundee Root Mean Squared Error (RMSE): 46.27902640824371
Whitfield Centre, Dundee Root Mean Squared Error (RMSE): 60.70383889187623
Housing Office East, Dundee Root Mean Squared Error (RMSE): 72.69204156438616
Gellatly Street Car Park, Dundee Root Mean Squared Error (RMSE): 99.10151718191786
Public Works Dept, Clepington Rd. Dundee Root Mean Squared Error (RMSE): 64.48960940301423
Marchbanks, Dundee Root Mean Squared Error (RMSE): 37.040941719948435
Olympia Multi-Storey Car Park, Dundee Root Mean Squared Error (RMSE): 58.66410484462479
South Tay Street Root Mean Squared Error (RMSE): 54.40839416359852
Ardler Complex, Dundee Root Mean Squared Error (RMSE): 18.94388348339981
Menziehill House, Dundee Root Mean Squared Error (RMSE): 23.632005765644756
Turriff House Rannoch Road, Dundee Root Mean Squared Error (RMSE): 13.719588149881037

c:\Users\balda\.pyenv\pyenv-win\versions\3.11.0\Lib\site-packages\torch\nn\modules\rnn.py:82: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1
  warnings.warn("dropout option adds dropout after all but last "

stations_to_compare = ['Greenmarket Multi Car Park, Dundee', 'Oakland Day Centre, Dundee', 
                       'Olympia Multi-Storey Car Park, Dundee', 'Housing Office East, Dundee',
                       'Dock Street, Dundee']

all_stations = set(weekly_predictions_dict.keys())

transfer_dict = weekly_predictions_dict_transfer.copy()
local_predictions_dict = weekly_predictions_dict.copy()
for station in all_stations:
    if station not in stations_to_compare:
        transfer_dict.pop(station, None)
        local_predictions_dict.pop(station, None)

# Define the number of rows for subplots
n_rows = len(transfer_dict)

# Create subplots
fig, axs = plt.subplots(n_rows, 1, figsize=(6, 15))

# Iterate over each station
for i, station in enumerate(transfer_dict.keys()):

    # Get transfer predictions
    transfer_data = transfer_dict[station]
    y_test = transfer_data['y_test'][-7:]
    transfer_y_pred = transfer_data['y_pred'][-7:]
    transfer_rmse = transfer_data['rmse']

    # Get locally trained predictions
    local_data = local_predictions_dict[station]
    local_y_pred = local_data['y_pred'][-7:]
    local_rmse = local_data['rmse']

    # Plot true values against the index of observations
    x = np.arange(len(y_test))
    axs[i].plot(x, y_test, color='blue', label='True values', linewidth=3)

    # Plot local predictions
    axs[i].plot(x, local_y_pred, color='green', linestyle=':', label='Predicted values (Local)', linewidth=3)

    # Plot transfer predictions
    axs[i].plot(x, transfer_y_pred, color='red', linestyle='--', label='Predicted values (Transfer)', linewidth=3)

    # Set title, labels, and legend for each subplot
    axs[i].set_title(f'{station.split()[0]} - Local RMSE: {local_rmse:.2f} | Transfer RMSE: {transfer_rmse:.2f}')
    axs[i].set_xlabel('Observation Index')
    axs[i].set_ylabel('Value')
    axs[i].legend(fontsize=7)

# Adjust layout and show plot
plt.tight_layout()
plt.subplots_adjust(top=0.95)
plt.suptitle('Dundee trained predictions (Local) vs Transfer learning from Palo Alto', fontsize=15)
plt.show()

	Site	Charging event	Date	Total kWh
0	Lochee Charging Hub, Dundee	7751157	2018-09-06	22.287
1	Greenmarket Multi Car Park, Dundee	7751142	2018-09-06	7.080
2	Queen Street Car Park, Broughty Ferry, Dundee	7751140	2018-09-06	8.220
3	Housing Office West, Dundee	7751138	2018-09-06	4.780
4	Housing Office West, Dundee	7751136	2018-09-06	7.080

Forecasting EV Charging Demand in Palo Alto (USA) with Long Short-Term Memory Network¶

Table of Contents¶

1. Introduction¶

2. Data processing¶

3. Model Implementation¶

3.1 Daily Model Implementation¶

3.2 Weekly Model Implementation¶

3.3 Monthly Model Implementation¶

4. Transfer Learning¶

5. Conclusion¶